{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "model.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyNk2ps3g59HqhwnWkHTy1Im",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/gagan3012/project-code-py/blob/master/model.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lsTTbIxoSXTc",
        "outputId": "bcdb7549-f96e-48c1-c75b-1122426f7211"
      },
      "source": [
        "lines = ['https://github.com/Garvit244/Leetcode',\r\n",
        "         'https://github.com/shichao-an/leetcode-python',\r\n",
        "         'https://github.com/algorhythms/LeetCode',\r\n",
        "         'https://github.com/wuduhren/leetcode-python',\r\n",
        "         'https://github.com/csujedihy/lc-all-solutions',\r\n",
        "         'https://github.com/vJechsmayr/PythonAlgorithms',\r\n",
        "         'https://github.com/HuberTRoy/leetCode',\r\n",
        "         'https://github.com/qiyuangong/leetcode',\r\n",
        "         'https://github.com/MTrajK/coding-problems',\r\n",
        "         'https://github.com/JushuangQiao/Python-LeetCode',\r\n",
        "         'https://github.com/Jack-Lee-Hiter/AlgorithmsByPython',\r\n",
        "         'https://github.com/sapanz/Hackerrank-Problem-Solving-Python-Solutions',\r\n",
        "         'https://github.com/arsho/Hackerrank_Python_Domain_Solutions',\r\n",
        "         'https://github.com/swapnanildutta/Hackerrank-Codes',\r\n",
        "         'https://github.com/markopuza/Competitive-programming-in-Python',\r\n",
        "         'https://github.com/deepaksood619/Python-Competitive-Programming',\r\n",
        "         'https://github.com/ndb796/Python-Competitive-Programming-Team-Notes',\r\n",
        "         'https://github.com/harshitbansal373/python',\r\n",
        "         'https://github.com/yashagrawal300/python-programs',\r\n",
        "         'https://github.com/bmegha98/Python-Practice',\r\n",
        "         'https://github.com/geekcomputers/Python',\r\n",
        "         'https://github.com/smilejay/python',\r\n",
        "         'https://github.com/yuzhoujr/leetcode',\r\n",
        "         'https://github.com/franklingu/leetcode-solutions',\r\n",
        "         'https://github.com/kumailn/Algorithms']\r\n",
        "\r\n",
        "len(lines)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "25"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 1
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8ArghELG6QZ5"
      },
      "source": [
        "from subprocess import call\r\n",
        "import math\r\n",
        "import os\r\n",
        "import csv\r\n",
        "csv_columns = ['text']\r\n",
        "\r\n",
        "\r\n",
        "for line in lines:\r\n",
        "    call(['git', 'clone', line.strip(), f'resources/{line.strip().split(\"/\")[-1]}'])\r\n",
        "\r\n",
        "json_data = []\r\n",
        "total_files = []\r\n",
        "count = 0\r\n",
        "\r\n",
        "for line in lines:\r\n",
        "    for currentpath, folders, files in os.walk(f'resources/{line.strip().split(\"/\")[-1]}'):\r\n",
        "        for file in files:\r\n",
        "            if file[-3:] == '.py':\r\n",
        "                print(file)\r\n",
        "                count += 1\r\n",
        "                total_files.append(os.path.join(currentpath, file))\r\n",
        "\r\n",
        "print('files: ', len(total_files))\r\n",
        "\r\n",
        "for file in total_files:\r\n",
        "    with open(file, \"r\") as f:\r\n",
        "        try:\r\n",
        "            t = f.readlines()\r\n",
        "        except UnicodeDecodeError:\r\n",
        "            print('DecoderError: ', file)\r\n",
        "        summary = ''.join(t)\r\n",
        "        summary = str(summary).strip()\r\n",
        "        bos_token = '<|title|>'\r\n",
        "        eos_token = '<|endoftext|>'\r\n",
        "        data = bos_token + summary + eos_token\r\n",
        "        json_data.append({'text': data})"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JHdBhEpR6cfD"
      },
      "source": [
        "with open(\"data.csv\", 'w') as csvfile:\r\n",
        "        writer = csv.DictWriter(csvfile, fieldnames=csv_columns)\r\n",
        "        writer.writeheader()\r\n",
        "        for data in json_data:\r\n",
        "            writer.writerow(data)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tH8MJYkm8Ldt"
      },
      "source": [
        "import pandas as pd\r\n",
        "\r\n",
        "df = pd.read_csv('/content/data.csv')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oeTMr6_W-o4x",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 203
        },
        "outputId": "3a623365-7b5c-4fb0-975d-2a8e5bf3a801"
      },
      "source": [
        "df.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>&lt;|title|&gt;'''\\nIn an N by N square grid, each c...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>&lt;|title|&gt;'''\\n\\tRemove the minimum number of i...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>&lt;|title|&gt;'''\\nGiven a stream of integers and a...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>&lt;|title|&gt;'''\\n    You are given an integer arr...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>&lt;|title|&gt;'''\\n\\tGiven a singly linked list, gr...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                text\n",
              "0  <|title|>'''\\nIn an N by N square grid, each c...\n",
              "1  <|title|>'''\\n\\tRemove the minimum number of i...\n",
              "2  <|title|>'''\\nGiven a stream of integers and a...\n",
              "3  <|title|>'''\\n    You are given an integer arr...\n",
              "4  <|title|>'''\\n\\tGiven a singly linked list, gr..."
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EAmmNuZOUqeY",
        "outputId": "e87655e1-5dc6-4aa4-b3d5-bb8beb95ca8a"
      },
      "source": [
        "from sklearn.model_selection import train_test_split\r\n",
        "\r\n",
        "train, eval = train_test_split(df, train_size=.9, random_state=2020)\r\n",
        "print(len(train))\r\n",
        "print(len(eval))\r\n",
        "\r\n",
        "train = train['text'].tolist()\r\n",
        "eval = eval['text'].tolist()\r\n",
        "\r\n",
        "\r\n",
        "with open('train_tmp.txt', 'w') as file_handle:\r\n",
        "  file_handle.write(str(train))\r\n",
        "\r\n",
        "with open('eval_tmp.txt', 'w') as file_handle:\r\n",
        "  file_handle.write(str(eval))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "5142\n",
            "572\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8JxD4TxQ-qCt"
      },
      "source": [
        "!git clone https://github.com/huggingface/transformers\r\n",
        "!pip install transformers\r\n",
        "!pip install datasets\r\n",
        "!pip install wandb"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-ojIGLxZU1eK"
      },
      "source": [
        "import os\r\n",
        "os.chdir(\"/content/transformers/examples/\")\r\n",
        "os.chdir(\"./language-modeling\")\r\n",
        "!pip install -r requirements.txt"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 105
        },
        "id": "9Z51w4XLXjh-",
        "outputId": "5ba7017c-1f7a-4feb-df0a-73f84bacf84d"
      },
      "source": [
        "import wandb\r\n",
        "wandb.login()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/javascript": [
              "\n",
              "        window._wandbApiKey = new Promise((resolve, reject) => {\n",
              "            function loadScript(url) {\n",
              "            return new Promise(function(resolve, reject) {\n",
              "                let newScript = document.createElement(\"script\");\n",
              "                newScript.onerror = reject;\n",
              "                newScript.onload = resolve;\n",
              "                document.body.appendChild(newScript);\n",
              "                newScript.src = url;\n",
              "            });\n",
              "            }\n",
              "            loadScript(\"https://cdn.jsdelivr.net/npm/postmate/build/postmate.min.js\").then(() => {\n",
              "            const iframe = document.createElement('iframe')\n",
              "            iframe.style.cssText = \"width:0;height:0;border:none\"\n",
              "            document.body.appendChild(iframe)\n",
              "            const handshake = new Postmate({\n",
              "                container: iframe,\n",
              "                url: 'https://wandb.ai/authorize'\n",
              "            });\n",
              "            const timeout = setTimeout(() => reject(\"Couldn't auto authenticate\"), 5000)\n",
              "            handshake.then(function(child) {\n",
              "                child.on('authorize', data => {\n",
              "                    clearTimeout(timeout)\n",
              "                    resolve(data)\n",
              "                });\n",
              "            });\n",
              "            })\n",
              "        });\n",
              "    "
            ],
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: You can find your API key in your browser here: https://wandb.ai/authorize\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "wandb: Paste an API key from your profile and hit enter: ··········\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: Appending key for api.wandb.ai to your netrc file: /root/.netrc\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yIpfn72NV9Md",
        "outputId": "c7e068df-bdac-4094-d8a7-ae5ac494c6b4"
      },
      "source": [
        "%env WANDB_PROJECT=project-code-py\r\n",
        "\r\n",
        "!python run_clm.py \\\r\n",
        "--model_type gpt2 \\\r\n",
        "--model_name_or_path gpt2 \\\r\n",
        "--train_file \"/content/train_tmp.txt\" \\\r\n",
        "--do_train \\\r\n",
        "--validation_file \"/content/eval_tmp.txt\" \\\r\n",
        "--do_eval \\\r\n",
        "--per_device_train_batch_size 1 \\\r\n",
        "--per_device_eval_batch_size 1 \\\r\n",
        "--save_steps -1 \\\r\n",
        "--num_train_epochs 5 \\\r\n",
        "--fp16 \\\r\n",
        "--output_dir=\"/content/model\" \\\r\n",
        "--report_to wandb "
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "env: WANDB_PROJECT=project-code-py\n",
            "2021-03-03 14:08:48.939262: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n",
            "03/03/2021 14:08:50 - WARNING - __main__ -   Process rank: -1, device: cuda:0, n_gpu: 1distributed training: False, 16-bits training: True\n",
            "03/03/2021 14:08:50 - INFO - __main__ -   Training/evaluation parameters TrainingArguments(output_dir=/content/model, overwrite_output_dir=False, do_train=True, do_eval=True, do_predict=False, evaluation_strategy=EvaluationStrategy.NO, prediction_loss_only=False, per_device_train_batch_size=1, per_device_eval_batch_size=1, gradient_accumulation_steps=1, eval_accumulation_steps=None, learning_rate=5e-05, weight_decay=0.0, adam_beta1=0.9, adam_beta2=0.999, adam_epsilon=1e-08, max_grad_norm=1.0, num_train_epochs=5.0, max_steps=-1, lr_scheduler_type=SchedulerType.LINEAR, warmup_steps=0, logging_dir=runs/Mar03_14-08-49_b2e9572afa44, logging_first_step=False, logging_steps=500, save_steps=-1, save_total_limit=None, no_cuda=False, seed=42, fp16=True, fp16_opt_level=O1, fp16_backend=auto, local_rank=-1, tpu_num_cores=None, tpu_metrics_debug=False, debug=False, dataloader_drop_last=False, eval_steps=500, dataloader_num_workers=0, past_index=-1, run_name=/content/model, disable_tqdm=False, remove_unused_columns=True, label_names=None, load_best_model_at_end=False, metric_for_best_model=None, greater_is_better=None, ignore_data_skip=False, sharded_ddp=False, deepspeed=None, label_smoothing_factor=0.0, adafactor=False, group_by_length=False, report_to=['wandb'], ddp_find_unused_parameters=None, dataloader_pin_memory=True, _n_gpu=1)\n",
            "03/03/2021 14:08:50 - WARNING - datasets.builder -   Using custom data configuration default-851166730c94ea0b\n",
            "03/03/2021 14:08:50 - WARNING - datasets.builder -   Reusing dataset text (/root/.cache/huggingface/datasets/text/default-851166730c94ea0b/0.0.0/44d63bd03e7e554f16131765a251f2d8333a5fe8a73f6ea3de012dbc49443691)\n",
            "[INFO|configuration_utils.py:449] 2021-03-03 14:08:50,192 >> loading configuration file https://huggingface.co/gpt2/resolve/main/config.json from cache at /root/.cache/huggingface/transformers/fc674cd6907b4c9e933cb42d67662436b89fa9540a1f40d7c919d0109289ad01.7d2e0efa5ca20cef4fb199382111e9d3ad96fd77b849e1d4bed13a66e1336f51\n",
            "[INFO|configuration_utils.py:485] 2021-03-03 14:08:50,192 >> Model config GPT2Config {\n",
            "  \"activation_function\": \"gelu_new\",\n",
            "  \"architectures\": [\n",
            "    \"GPT2LMHeadModel\"\n",
            "  ],\n",
            "  \"attn_pdrop\": 0.1,\n",
            "  \"bos_token_id\": 50256,\n",
            "  \"embd_pdrop\": 0.1,\n",
            "  \"eos_token_id\": 50256,\n",
            "  \"gradient_checkpointing\": false,\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"layer_norm_epsilon\": 1e-05,\n",
            "  \"model_type\": \"gpt2\",\n",
            "  \"n_ctx\": 1024,\n",
            "  \"n_embd\": 768,\n",
            "  \"n_head\": 12,\n",
            "  \"n_inner\": null,\n",
            "  \"n_layer\": 12,\n",
            "  \"n_positions\": 1024,\n",
            "  \"resid_pdrop\": 0.1,\n",
            "  \"summary_activation\": null,\n",
            "  \"summary_first_dropout\": 0.1,\n",
            "  \"summary_proj_to_labels\": true,\n",
            "  \"summary_type\": \"cls_index\",\n",
            "  \"summary_use_proj\": true,\n",
            "  \"task_specific_params\": {\n",
            "    \"text-generation\": {\n",
            "      \"do_sample\": true,\n",
            "      \"max_length\": 50\n",
            "    }\n",
            "  },\n",
            "  \"transformers_version\": \"4.3.3\",\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50257\n",
            "}\n",
            "\n",
            "[INFO|configuration_utils.py:449] 2021-03-03 14:08:50,210 >> loading configuration file https://huggingface.co/gpt2/resolve/main/config.json from cache at /root/.cache/huggingface/transformers/fc674cd6907b4c9e933cb42d67662436b89fa9540a1f40d7c919d0109289ad01.7d2e0efa5ca20cef4fb199382111e9d3ad96fd77b849e1d4bed13a66e1336f51\n",
            "[INFO|configuration_utils.py:485] 2021-03-03 14:08:50,210 >> Model config GPT2Config {\n",
            "  \"activation_function\": \"gelu_new\",\n",
            "  \"architectures\": [\n",
            "    \"GPT2LMHeadModel\"\n",
            "  ],\n",
            "  \"attn_pdrop\": 0.1,\n",
            "  \"bos_token_id\": 50256,\n",
            "  \"embd_pdrop\": 0.1,\n",
            "  \"eos_token_id\": 50256,\n",
            "  \"gradient_checkpointing\": false,\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"layer_norm_epsilon\": 1e-05,\n",
            "  \"model_type\": \"gpt2\",\n",
            "  \"n_ctx\": 1024,\n",
            "  \"n_embd\": 768,\n",
            "  \"n_head\": 12,\n",
            "  \"n_inner\": null,\n",
            "  \"n_layer\": 12,\n",
            "  \"n_positions\": 1024,\n",
            "  \"resid_pdrop\": 0.1,\n",
            "  \"summary_activation\": null,\n",
            "  \"summary_first_dropout\": 0.1,\n",
            "  \"summary_proj_to_labels\": true,\n",
            "  \"summary_type\": \"cls_index\",\n",
            "  \"summary_use_proj\": true,\n",
            "  \"task_specific_params\": {\n",
            "    \"text-generation\": {\n",
            "      \"do_sample\": true,\n",
            "      \"max_length\": 50\n",
            "    }\n",
            "  },\n",
            "  \"transformers_version\": \"4.3.3\",\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50257\n",
            "}\n",
            "\n",
            "[INFO|tokenization_utils_base.py:1786] 2021-03-03 14:08:50,284 >> loading file https://huggingface.co/gpt2/resolve/main/vocab.json from cache at /root/.cache/huggingface/transformers/684fe667923972fb57f6b4dcb61a3c92763ad89882f3da5da9866baf14f2d60f.c7ed1f96aac49e745788faa77ba0a26a392643a50bb388b9c04ff469e555241f\n",
            "[INFO|tokenization_utils_base.py:1786] 2021-03-03 14:08:50,284 >> loading file https://huggingface.co/gpt2/resolve/main/merges.txt from cache at /root/.cache/huggingface/transformers/c0c761a63004025aeadd530c4c27b860ec4ecbe8a00531233de21d865a402598.5d12962c5ee615a4c803841266e9c3be9a691a924f72d395d3a6c6c81157788b\n",
            "[INFO|tokenization_utils_base.py:1786] 2021-03-03 14:08:50,284 >> loading file https://huggingface.co/gpt2/resolve/main/tokenizer.json from cache at /root/.cache/huggingface/transformers/16a2f78023c8dc511294f0c97b5e10fde3ef9889ad6d11ffaa2a00714e73926e.cf2d0ecb83b6df91b3dbb53f1d1e4c311578bfd3aa0e04934215a49bf9898df0\n",
            "[INFO|modeling_utils.py:1027] 2021-03-03 14:08:50,365 >> loading weights file https://huggingface.co/gpt2/resolve/main/pytorch_model.bin from cache at /root/.cache/huggingface/transformers/752929ace039baa8ef70fe21cdf9ab9445773d20e733cf693d667982e210837e.323c769945a351daa25546176f8208b3004b6f563438a7603e7932bae9025925\n",
            "[INFO|modeling_utils.py:1143] 2021-03-03 14:08:55,229 >> All model checkpoint weights were used when initializing GPT2LMHeadModel.\n",
            "\n",
            "[INFO|modeling_utils.py:1152] 2021-03-03 14:08:55,229 >> All the weights of GPT2LMHeadModel were initialized from the model checkpoint at gpt2.\n",
            "If your task is similar to the task the model of the checkpoint was trained on, you can already use GPT2LMHeadModel for predictions without further training.\n",
            "[WARNING|tokenization_utils_base.py:3213] 2021-03-03 14:09:09,329 >> Token indices sequence length is longer than the specified maximum sequence length for this model (6955447 > 1024). Running this sequence through the model will result in indexing errors\n",
            "03/03/2021 14:09:09 - WARNING - datasets.arrow_dataset -   Loading cached processed dataset at /root/.cache/huggingface/datasets/text/default-851166730c94ea0b/0.0.0/44d63bd03e7e554f16131765a251f2d8333a5fe8a73f6ea3de012dbc49443691/cache-a09e896812ed5c6b.arrow\n",
            "03/03/2021 14:09:10 - WARNING - datasets.arrow_dataset -   Loading cached processed dataset at /root/.cache/huggingface/datasets/text/default-851166730c94ea0b/0.0.0/44d63bd03e7e554f16131765a251f2d8333a5fe8a73f6ea3de012dbc49443691/cache-a1f276bd548ce432.arrow\n",
            "03/03/2021 14:09:15 - WARNING - datasets.arrow_dataset -   Loading cached processed dataset at /root/.cache/huggingface/datasets/text/default-851166730c94ea0b/0.0.0/44d63bd03e7e554f16131765a251f2d8333a5fe8a73f6ea3de012dbc49443691/cache-344e59f7ee2764ea.arrow\n",
            "03/03/2021 14:09:15 - WARNING - datasets.arrow_dataset -   Loading cached processed dataset at /root/.cache/huggingface/datasets/text/default-851166730c94ea0b/0.0.0/44d63bd03e7e554f16131765a251f2d8333a5fe8a73f6ea3de012dbc49443691/cache-691e4d3d63f0bacc.arrow\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "[INFO|trainer.py:432] 2021-03-03 14:09:18,280 >> The following columns in the training set don't have a corresponding argument in `GPT2LMHeadModel.forward` and have been ignored: .\n",
            "[INFO|trainer.py:432] 2021-03-03 14:09:18,281 >> The following columns in the evaluation set don't have a corresponding argument in `GPT2LMHeadModel.forward` and have been ignored: .\n",
            "[INFO|trainer.py:348] 2021-03-03 14:09:18,281 >> Using amp fp16 backend\n",
            "[INFO|trainer.py:837] 2021-03-03 14:09:18,285 >> ***** Running training *****\n",
            "[INFO|trainer.py:838] 2021-03-03 14:09:18,285 >>   Num examples = 6792\n",
            "[INFO|trainer.py:839] 2021-03-03 14:09:18,285 >>   Num Epochs = 5\n",
            "[INFO|trainer.py:840] 2021-03-03 14:09:18,285 >>   Instantaneous batch size per device = 1\n",
            "[INFO|trainer.py:841] 2021-03-03 14:09:18,285 >>   Total train batch size (w. parallel, distributed & accumulation) = 1\n",
            "[INFO|trainer.py:842] 2021-03-03 14:09:18,285 >>   Gradient Accumulation steps = 1\n",
            "[INFO|trainer.py:843] 2021-03-03 14:09:18,285 >>   Total optimization steps = 33960\n",
            "[INFO|integrations.py:546] 2021-03-03 14:09:18,287 >> Automatic Weights & Biases logging enabled, to disable set os.environ[\"WANDB_DISABLED\"] = \"true\"\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mgagan3012\u001b[0m (use `wandb login --relogin` to force relogin)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "2021-03-03 14:09:19.452432: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.10.21\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Syncing run \u001b[33m/content/model\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: ⭐️ View project at \u001b[34m\u001b[4mhttps://wandb.ai/gagan3012/project-code-py\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run at \u001b[34m\u001b[4mhttps://wandb.ai/gagan3012/project-code-py/runs/32ariuue\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run data is saved locally in /content/transformers/examples/language-modeling/wandb/run-20210303_140918-32ariuue\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run `wandb offline` to turn off syncing.\n",
            "\n",
            "  0%|          | 0/33960 [00:00<?, ?it/s]/usr/local/lib/python3.7/dist-packages/torch/optim/lr_scheduler.py:136: UserWarning: Detected call of `lr_scheduler.step()` before `optimizer.step()`. In PyTorch 1.1.0 and later, you should call them in the opposite order: `optimizer.step()` before `lr_scheduler.step()`.  Failure to do this will result in PyTorch skipping the first value of the learning rate schedule. See more details at https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate\n",
            "  \"https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate\", UserWarning)\n",
            "  1%|▏         | 500/33960 [02:14<3:00:40,  3.09it/s]{'loss': 2.7045, 'learning_rate': 4.926383981154299e-05, 'epoch': 0.07}\n",
            "  3%|▎         | 1000/33960 [04:30<2:58:29,  3.08it/s]{'loss': 2.6319, 'learning_rate': 4.852767962308599e-05, 'epoch': 0.15}\n",
            "{'loss': 2.5401, 'learning_rate': 4.779151943462898e-05, 'epoch': 0.22}\n",
            "                                                      \n",
            "                                                      \n",
            "  9%|▉         | 3000/33960 [13:33<2:47:10,  3.09it/s]{'loss': 2.4634, 'learning_rate': 4.558303886925796e-05, 'epoch': 0.44}\n",
            "{'loss': 2.5514, 'learning_rate': 4.484687868080095e-05, 'epoch': 0.52}\n",
            "{'loss': 2.3272, 'learning_rate': 4.411071849234394e-05, 'epoch': 0.59}\n",
            "                                                      {'loss': 2.4235, 'learning_rate': 4.337455830388692e-05, 'epoch': 0.66}\n",
            "{'loss': 2.3926, 'learning_rate': 4.263839811542992e-05, 'epoch': 0.74}\n",
            "{'loss': 2.4661, 'learning_rate': 4.190223792697291e-05, 'epoch': 0.81}\n",
            " 18%|█▊        | 6000/33960 [27:10<2:30:39,  3.09it/s]{'loss': 2.456, 'learning_rate': 4.11660777385159e-05, 'epoch': 0.88}\n",
            "{'loss': 2.4138, 'learning_rate': 4.042991755005889e-05, 'epoch': 0.96}\n",
            "{'loss': 2.318, 'learning_rate': 3.969375736160189e-05, 'epoch': 1.03}\n",
            "                                                      {'loss': 2.3313, 'learning_rate': 3.895759717314488e-05, 'epoch': 1.1}\n",
            "{'loss': 2.3808, 'learning_rate': 3.822143698468787e-05, 'epoch': 1.18}\n",
            "{'loss': 2.3253, 'learning_rate': 3.748527679623086e-05, 'epoch': 1.25}\n",
            "{'loss': 2.2931, 'learning_rate': 3.6749116607773855e-05, 'epoch': 1.33}\n",
            "{'loss': 2.2898, 'learning_rate': 3.6012956419316846e-05, 'epoch': 1.4}\n",
            "{'loss': 2.1639, 'learning_rate': 3.5276796230859836e-05, 'epoch': 1.47}\n",
            " 31%|███       | 10500/33960 [47:36<2:05:22,  3.12it/s]{'loss': 2.2034, 'learning_rate': 3.454063604240283e-05, 'epoch': 1.55}\n",
            "{'loss': 2.2413, 'learning_rate': 3.380447585394582e-05, 'epoch': 1.62}\n",
            "{'loss': 2.3314, 'learning_rate': 3.3068315665488814e-05, 'epoch': 1.69}\n",
            "{'loss': 2.3188, 'learning_rate': 3.2332155477031804e-05, 'epoch': 1.77}\n",
            "{'loss': 2.3464, 'learning_rate': 3.15959952885748e-05, 'epoch': 1.84}\n",
            "{'loss': 2.2622, 'learning_rate': 3.0859835100117785e-05, 'epoch': 1.91}\n",
            "{'loss': 2.3083, 'learning_rate': 3.0123674911660775e-05, 'epoch': 1.99}\n",
            "{'loss': 2.167, 'learning_rate': 2.938751472320377e-05, 'epoch': 2.06}\n",
            "{'loss': 2.1884, 'learning_rate': 2.865135453474676e-05, 'epoch': 2.13}\n",
            "{'loss': 2.1836, 'learning_rate': 2.7915194346289753e-05, 'epoch': 2.21}\n",
            "{'loss': 2.2923, 'learning_rate': 2.7179034157832743e-05, 'epoch': 2.28}\n",
            "{'loss': 2.2081, 'learning_rate': 2.6442873969375737e-05, 'epoch': 2.36}\n",
            " 49%|████▊     | 16500/33960 [1:14:49<1:33:12,  3.12it/s]{'loss': 2.24, 'learning_rate': 2.5706713780918727e-05, 'epoch': 2.43}\n",
            " 50%|█████     | 17000/33960 [1:17:05<1:30:56,  3.11it/s]{'loss': 2.1503, 'learning_rate': 2.497055359246172e-05, 'epoch': 2.5}\n",
            " 52%|█████▏    | 17500/33960 [1:19:21<1:28:41,  3.09it/s]{'loss': 2.1941, 'learning_rate': 2.4234393404004714e-05, 'epoch': 2.58}\n",
            "{'loss': 2.194, 'learning_rate': 2.3498233215547705e-05, 'epoch': 2.65}\n",
            "{'loss': 2.2143, 'learning_rate': 2.27620730270907e-05, 'epoch': 2.72}\n",
            " 56%|█████▌    | 19000/33960 [1:26:10<1:20:25,  3.10it/s]{'loss': 2.2255, 'learning_rate': 2.202591283863369e-05, 'epoch': 2.8}\n",
            " 57%|█████▋    | 19500/33960 [1:28:27<1:18:25,  3.07it/s]{'loss': 2.2729, 'learning_rate': 2.128975265017668e-05, 'epoch': 2.87}\n",
            "{'loss': 2.2286, 'learning_rate': 2.055359246171967e-05, 'epoch': 2.94}\n",
            " 60%|██████    | 20500/33960 [1:32:59<1:12:08,  3.11it/s]{'loss': 2.1893, 'learning_rate': 1.9817432273262663e-05, 'epoch': 3.02}\n",
            " 62%|██████▏   | 21000/33960 [1:35:15<1:09:41,  3.10it/s]{'loss': 2.2082, 'learning_rate': 1.9081272084805653e-05, 'epoch': 3.09}\n",
            "{'loss': 2.1539, 'learning_rate': 1.8345111896348647e-05, 'epoch': 3.17}\n",
            "{'loss': 2.1413, 'learning_rate': 1.7608951707891637e-05, 'epoch': 3.24}\n",
            " 66%|██████▋   | 22500/33960 [1:42:04<1:02:22,  3.06it/s]{'loss': 2.2011, 'learning_rate': 1.687279151943463e-05, 'epoch': 3.31}\n",
            " 68%|██████▊   | 23000/33960 [1:44:20<59:06,  3.09it/s]{'loss': 2.1113, 'learning_rate': 1.613663133097762e-05, 'epoch': 3.39}\n",
            " 69%|██████▉   | 23500/33960 [1:46:36<56:27,  3.09it/s]{'loss': 2.1736, 'learning_rate': 1.5400471142520615e-05, 'epoch': 3.46}\n",
            "{'loss': 2.3318, 'learning_rate': 1.4664310954063604e-05, 'epoch': 3.53}\n",
            "{'loss': 2.2084, 'learning_rate': 1.3928150765606596e-05, 'epoch': 3.61}\n",
            "                                                       {'loss': 2.1434, 'learning_rate': 1.3191990577149588e-05, 'epoch': 3.68}\n",
            "                                                       \n",
            "{'loss': 2.1764, 'learning_rate': 1.1719670200235572e-05, 'epoch': 3.83}\n",
            "{'loss': 2.1553, 'learning_rate': 1.0983510011778564e-05, 'epoch': 3.9}\n",
            "{'loss': 2.1203, 'learning_rate': 1.0247349823321556e-05, 'epoch': 3.98}\n",
            "{'loss': 2.1366, 'learning_rate': 9.511189634864546e-06, 'epoch': 4.05}\n",
            "{'loss': 2.1054, 'learning_rate': 8.775029446407538e-06, 'epoch': 4.12}\n",
            "{'loss': 2.1389, 'learning_rate': 8.03886925795053e-06, 'epoch': 4.2}\n",
            "{'loss': 2.1825, 'learning_rate': 7.302709069493523e-06, 'epoch': 4.27}\n",
            "                                                       \n",
            "{'loss': 2.2047, 'learning_rate': 5.830388692579505e-06, 'epoch': 4.42}\n",
            "{'loss': 2.2576, 'learning_rate': 5.094228504122497e-06, 'epoch': 4.49}\n",
            "{'loss': 2.1254, 'learning_rate': 4.3580683156654886e-06, 'epoch': 4.56}\n",
            "{'loss': 2.2021, 'learning_rate': 3.621908127208481e-06, 'epoch': 4.64}\n",
            "{'loss': 2.1433, 'learning_rate': 2.8857479387514726e-06, 'epoch': 4.71}\n",
            "{'loss': 2.1185, 'learning_rate': 2.149587750294464e-06, 'epoch': 4.79}\n",
            " 97%|█████████▋| 33000/33960 [2:29:41<05:10,  3.09it/s]\n",
            " 99%|█████████▊| 33500/33960 [2:31:57<02:28,  3.09it/s]{'loss': 2.0775, 'learning_rate': 6.772673733804477e-07, 'epoch': 4.93}\n",
            "100%|██████████| 33960/33960 [2:34:02<00:00,  3.69it/s][INFO|trainer.py:1007] 2021-03-03 16:43:22,769 >> \n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "                                                       \n",
            "100%|██████████| 33960/33960 [2:34:02<00:00,  3.67it/s]\n",
            "[INFO|trainer.py:1408] 2021-03-03 16:43:22,791 >> Saving model checkpoint to /content/model\n",
            "[INFO|configuration_utils.py:304] 2021-03-03 16:43:22,794 >> Configuration saved in /content/model/config.json\n",
            "[INFO|modeling_utils.py:817] 2021-03-03 16:43:24,480 >> Model weights saved in /content/model/pytorch_model.bin\n",
            "Traceback (most recent call last):\n",
            "  File \"run_clm.py\", line 406, in <module>\n",
            "    main()\n",
            "  File \"run_clm.py\", line 380, in main\n",
            "    trainer.log_metrics(\"train\", metrics)\n",
            "AttributeError: 'Trainer' object has no attribute 'log_metrics'\n",
            "\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Waiting for W&B process to finish, PID 537\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Program failed with code 1.  Press ctrl-c to abort syncing.\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                                                                                \n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Find user logs for this run at: /content/transformers/examples/language-modeling/wandb/run-20210303_140918-32ariuue/logs/debug.log\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Find internal logs for this run at: /content/transformers/examples/language-modeling/wandb/run-20210303_140918-32ariuue/logs/debug-internal.log\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run summary:\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                                        _runtime 9244\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                                      _timestamp 1614789802\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                                           _step 33960\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                                      train/loss 2.0775\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                             train/learning_rate 0.0\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                                     train/epoch 5.0\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                             train/train_runtime 9244.4843\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                  train/train_samples_per_second 3.674\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                                train/total_flos 25964395804753920\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run history:\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                         _runtime ▁▁▁▂▂▂▂▂▂▃▃▃▃▃▄▄▄▄▄▄▅▅▅▅▅▅▆▆▆▆▆▇▇▇▇▇▇███\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                       _timestamp ▁▁▁▂▂▂▂▂▂▃▃▃▃▃▄▄▄▄▄▄▅▅▅▅▅▅▆▆▆▆▆▇▇▇▇▇▇███\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                            _step ▁▁▁▂▂▂▂▂▂▃▃▃▃▃▃▄▄▄▄▄▅▅▅▅▅▅▆▆▆▆▆▇▇▇▇▇▇███\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                       train/loss █▇▆▅▆▅▅▅▄▄▄▃▂▄▄▃▂▂▃▃▂▂▃▃▂▂▂▁▄▂▁▂▂▁▂▂▃▂▁▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:              train/learning_rate ███▇▇▇▇▇▇▆▆▆▆▆▆▅▅▅▅▅▅▄▄▄▄▄▃▃▃▃▃▂▂▂▂▂▂▁▁▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                      train/epoch ▁▁▁▂▂▂▂▂▂▃▃▃▃▃▄▄▄▄▄▄▅▅▅▅▅▅▆▆▆▆▆▇▇▇▇▇▇███\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:              train/train_runtime ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:   train/train_samples_per_second ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                 train/total_flos ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Synced 5 W&B file(s), 1 media file(s), 0 artifact file(s) and 0 other file(s)\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Synced \u001b[33m/content/model\u001b[0m: \u001b[34mhttps://wandb.ai/gagan3012/project-code-py/runs/32ariuue\u001b[0m\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NXsM18Oe-1hf"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}